package com.googlecode.quillen.application;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.lang.StringUtils;

import java.io.*;
import java.util.*;
import java.util.concurrent.Semaphore;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;

import com.googlecode.quillen.util.*;
import static com.googlecode.quillen.util.Utils.logInfo;
import static com.googlecode.quillen.util.Utils.logError;
import static com.googlecode.quillen.util.Utils.logWarning;
import com.googlecode.quillen.domain.*;
import com.googlecode.quillen.service.ChunkService;
import com.googlecode.quillen.service.ShadowFileService;
import com.googlecode.quillen.service.SnapshotService;

/**
 * Created by IntelliJ IDEA.
 * User: greg
 * Date: Dec 2, 2008
 * Time: 9:13:27 PM
 */
public class BackupImpl implements Backup
{
    private static final Log LOG = LogFactory.getLog(BackupImpl.class);
    private static final int MAX_SNAPSHOT_NAME_BYTES = 1009;    // So it can fit in attribute value with appended -yyyyMMddHHmmss
    private static final int MAX_FILENAME_BYTES = 1024;         // So it can fit in attribute values

    private final ChunkService chunkService;
    private final ShadowFileService shadowFileService;
    private final SnapshotService snapshotService;

    // TODO: allow no-follow of symlinks (not easy in java)

    public BackupImpl(final ChunkService chunkService,
                      final ShadowFileService shadowFileService,
                      final SnapshotService snapshotService)
    {
        this.chunkService = chunkService;
        this.shadowFileService = shadowFileService;
        this.snapshotService = snapshotService;
    }

    public void backup(final String base,
                       final String path,
                       final Collection<String> snapshots,
                       final ResultConsumer<FileInfo> consumer)
            throws IOException, WorkQueueAbortedException, ObjectStorageException, NoSuchAlgorithmException, AttributeStorageException, InterruptedException, ParseException
    {
        backupHelper(true, base, Arrays.asList(path), snapshots, consumer);
    }

    public void backup(final String base,
                       final Collection<String> paths,
                       final Collection<String> snapshots,
                       final ResultConsumer<FileInfo> consumer)
            throws IOException, WorkQueueAbortedException, ObjectStorageException, NoSuchAlgorithmException, AttributeStorageException, InterruptedException, ParseException
    {
        backupHelper(false, base, paths, snapshots, consumer);
    }

    private void backupHelper(final boolean relative,
                              String base,
                              final Collection<String> paths,
                              final Collection<String> snapshots,
                              final ResultConsumer<FileInfo> consumer)
            throws IOException, WorkQueueAbortedException, ObjectStorageException, NoSuchAlgorithmException, AttributeStorageException, InterruptedException, ParseException
    {
        // Make sure the base path ends with "/", so "" will become "/" and "/Users/greg" will become "/Users/greg/"
        //
        if (!base.endsWith(File.separator))
        {
            base += File.separator;
        }
        final String basePath = base;

        // Get a list of file handles for all given paths.  If relative is true, then each path is relative to base path
        //
        final ArrayList<File> pathFiles = new ArrayList<File>(paths.size());
        for (String path : paths)
        {
            File file = relative ? new File(basePath + StringUtils.defaultString(path)) : new File(path);

            if (file.exists())
            {
                pathFiles.add(file);
            }
        }

        if (pathFiles.isEmpty())
        {
            logInfo(LOG, "nothing to backup");

            return;
        }

        // Enforce constraints on snapshot names, and make sure a non-empty snapshot doesn't already exist with that name
        // If an empty version of the snapshot exists, that means it's leftover from a previous attempt
        //
        for (String snapshotName : snapshots)
        {
            if (snapshotName.getBytes("utf-8").length > MAX_SNAPSHOT_NAME_BYTES)
            {
                throw new IllegalArgumentException(String.format("snapshot name must be less than %d bytes (utf-8): %s", MAX_SNAPSHOT_NAME_BYTES, snapshotName));
            }

            Snapshot snapshot = snapshotService.get(snapshotName);
            if (snapshot != null && snapshot.getFiles() > 0)
            {
                throw new IllegalArgumentException(String.format("snapshot %s already exists with %d files and backup date %s", snapshotName, snapshot.getFiles(), snapshot.getDate()));
            }
        }

        // Put an empty snapshots as placeholders, so shadow files aren't pointing to non-existent snapshots
        //
        Date now = new Date();
        for (String snapshot : snapshots)
        {
            snapshotService.put(new Snapshot(snapshot, now, 0, 0));
        }

        // Use an exception handler that will abort when there's an error
        //
        ExceptionHandler exceptionHandler = new ExceptionHandler()
        {
            public boolean handleException(Exception ex)
            {
                logError(LOG, ex, consumer, "error while backing up");

                return true;
            }
        };

        // Use 3 work queues: one to process files, one to put shadow files, and one to put chunks
        //
        // TODO: tune these work queue parameters
        final WorkQueue fileWorkQueue = new WorkQueue(10, 20, exceptionHandler);
        final WorkQueue shadowFileWorkQueue = new WorkQueue(10, 50, exceptionHandler);
        final WorkQueue chunkWorkQueue = new WorkQueue(10, 10, exceptionHandler);

        final MutexProvider mutexProvider = new MutexProvider();
        final String tempFilePrefix = TempFileUtils.getPrefix();
        final ShadowFileBatch shadowFileBatch = new ShadowFileBatch(shadowFileService, shadowFileWorkQueue, snapshots, consumer);
        
        long totalSize = 0;
        int totalFiles = 0;
        File lastFile = null;

        try
        {
            // SortedFileList will sort the files by size so that we process smaller files first (get more done faster)
            // and also to ensure grouping equivalent files (same contents) that are represented by one shadow file
            //
            SortedFileList list = new SortedFileList(pathFiles);
            try
            {
                for (final File file : list)
                {
                    if (file.exists() && file.isFile() && file != lastFile)
                    {
                        fileWorkQueue.enqueue(new WorkItem()
                        {
                            public void run() throws Exception
                            {
                                backupFile(basePath, file, chunkWorkQueue, mutexProvider, shadowFileBatch, tempFilePrefix);
                            }
                        });

                        totalSize += file.length();
                        ++totalFiles;
                        lastFile = file;
                    }
                }
            }
            finally
            {
                list.close();
            }

            fileWorkQueue.freezeAndBlockUntilEmpty();

            // Put any remaining shadow files
            //
            shadowFileBatch.flush(0);

            shadowFileWorkQueue.freezeAndBlockUntilEmpty();
            chunkWorkQueue.freezeAndBlockUntilEmpty();

            // Put the snapshots with the new file count, size, and time stamp
            //
            long totalTime = new Date().getTime() - now.getTime();
            now = new Date();
            for (String snapshot : snapshots)
            {
                snapshotService.put(new Snapshot(snapshot, now, totalFiles, totalSize));
            }

            logInfo(LOG, "backed up %d files with %d bytes to snapshot %s (~%.1f KB/s) (~%.1f files/s)",
                    totalFiles,
                    totalSize,
                    StringUtils.join(snapshots, ", "),
                    (double) totalSize / (double) totalTime,
                    (double) totalFiles / (double) totalTime * 1000.0);
        }
        finally
        {
            // Abort any non-empty work queues (like if there was an error) and delete temp files
            //
            // TODO: what if an error occured before any items were enqueued...  need a better solution (like abort queues that were started but not finished)
            if (!fileWorkQueue.isEmpty())
            {
                fileWorkQueue.abort();
            }

            if (!shadowFileWorkQueue.isEmpty())
            {
                shadowFileWorkQueue.abort();
            }

            if (!chunkWorkQueue.isEmpty())
            {
                chunkWorkQueue.abort();
            }

            TempFileUtils.deleteAll(tempFilePrefix);
        }
    }

    private void backupFile(final String base,
                            final File file,
                            final WorkQueue chunkWorkQueue,
                            final MutexProvider mutexProvider,
                            final ShadowFileBatch shadowFileBatch,
                            final String tempFilePrefix)
            throws NoSuchAlgorithmException, IOException, WorkQueueAbortedException, ObjectStorageException, AttributeStorageException, InterruptedException
    {
        // Strip the base path off the beginning of the filename
        //
        final String filename = StringUtils.removeStart(file.getAbsolutePath(), base);

        // Constrain filenames to 100 chars so they'll fit in SimpleDB attribute values
        //
        if (filename.getBytes("utf-8").length > MAX_FILENAME_BYTES)
        {
            throw new IllegalArgumentException(String.format("file name must be less than %d bytes (utf-8): %s", MAX_FILENAME_BYTES, filename));
        }

        // Make a temp copy of the file and process that, in case it is being written to or accessed otherwise
        // However, if we can't make a copy (not enough space or some other error), just process the file itself
        //
        File fileToProcess;
        final File tempFile = TempFileUtils.createTempFile(tempFilePrefix);
        try
        {
            FileUtils.copyFile(file, tempFile);
            fileToProcess = tempFile;
        }
        catch (IOException e)
        {
            logWarning(LOG,
                       "failed to create temp copy at %s (%s), continuing with file %s.  This can be dangerous if the file is changing during backup!",
                       tempFile.getAbsolutePath(),
                       e.getMessage(),
                       file.getAbsolutePath());
            
            FileUtils.deleteQuietly(tempFile);
            fileToProcess = file;
        }

        if (fileToProcess.exists())
        {
            // Calculate the shadow key from the file contents
            //
            final String shadowKey = shadowFileService.createShadowKey(new FileInputStream(fileToProcess));

            // Make sure we're the only one processing this shadow file
            //
            synchronized (mutexProvider.getMutex("shadow-" + shadowKey))
            {
                // Get the shadow file (from the batch if it's already there, from the attribute storage if it's there, or
                // a new instance) and add it to the current batch.  This may cause a flush if the current batch is full
                //
                final ShadowFile shadowFile = shadowFileBatch.add(shadowKey, filename, fileToProcess.length());

                if (shadowFile != null)
                {
                    // Read the file in chunks, put those chunks (associate them with this shadow file).  Don't re-upload existing chunks
                    //
                    final StreamChunker chunker = new StreamChunker(new FileInputStream(fileToProcess));
                    final Semaphore sem = new Semaphore(0);

                    Chunk c;
                    while ((c = chunker.getNextChunk()) != null)
                    {
                        final Chunk chunk = c;

                        shadowFile.addChunk(chunk);

                        chunkWorkQueue.enqueue(new WorkItem()
                        {
                            public void run() throws ObjectStorageException, AttributeStorageException, IOException
                            {
                                synchronized (mutexProvider.getMutex("chunk-" + chunk.getKey()))
                                {
                                    chunkService.put(shadowKey, chunk);
                                }

                                sem.release();
                            }
                        });
                    }

                    // As soon as all the chunks are put, signal to ShadowFileBatch that we're done
                    //
                    sem.acquire(shadowFile.getChunkKeys().size());
                    shadowFileBatch.signalDone(shadowKey);
                }
            }

            FileUtils.deleteQuietly(tempFile);
        }
        else
        {
            logWarning(LOG, "file %s seems to have disappeared!", fileToProcess.toString());
        }
    }
}
